import requests
import pymongo
import time
import log_helper
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

myclient = pymongo.MongoClient("mongodb://82.157.127.245:27999/")
mydb = myclient["weibo"]
mycollection = mydb["hotsearch"]

request_headers = {
    "accept": "application/json, text/plain, */*",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36",
    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
    "referer": "https://weibo.com/u/7203149652/home?wvr=5",
    "connection": "close",
    "cookie": "SINAGLOBAL=5188697282285.41.1591635317397; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhwZJpoNjEdQDUxBBSkLQ1w5JpX5KMhUgL.FoMEehepSh.cSKz2dJLoIED0qgxrUP.LxKnLB-qLBK-LxKqL1h5L12-LxKBLBo.L1hnLxK-L122L1--t; ALF=1666505945; SSOLoginState=1634969946; SCF=Ai2Z0Av6-LNog9gdRY2nsteOJC6e2CE2SpTKVU8Z4qEGFUB6h5V0u3Djc37MdZBWfpzffkKHZkri-9XqjP_Cx2g.; SUB=_2A25Md9kKDeRhGeFM61EQ9CfKzj6IHXVvBU3CrDV8PUNbmtB-LXLHkW9NQOfUnl0FyL09NY6T8ltfzg0IAoK2sE2n; XSRF-TOKEN=m_e8u5KyXEuBW7-vmirHQHi5; _s_tentry=weibo.com; Apache=8662431052348.949.1634969987434; ULV=1634969987471:11:11:6:8662431052348.949.1634969987434:1634887816113; WBPSESS=r95T7PN5rvAw4EQtZLfQyYcGGjy6n_9gsJTohhO1sUMP69xNnpjrDvz0x6VftAVIaSqVHKSd9Dh27-GSzInDUwb2xjrdg8eZyR3P2QAOFyhdnADVNYIs5CkFnQKEZMK8"


}
# hotSearch:get

hot_search_url = "https://weibo.com/ajax/side/hotSearch"

hot_search_resp = requests.session().get(hot_search_url, headers=request_headers, verify=False)
realtime_list = hot_search_resp.json().get('data').get('realtime')

search_url = "https://weibo.com/ajax/search/all"

time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())


page = 1
blog_cards = []

while True:

    params = {
        "containerid": "100103type=1&q=#恒大支付8350万美元债利息#&t=3",
        "page": page,
        "count": 20
    }
    try:
        search_resp = requests.session().get(search_url, headers=request_headers, params=params, verify=False)
    except requests.exceptions.ConnectionError:
        log_helper.error("话题的第{}页获取失败".format(page))

    status = search_resp.json().get("ok")
    if status < 0:
        log_helper.error("cookies失效")
        break;
    if search_resp.json().get("data") is None:
        continue

    cards = search_resp.json().get("data").get("cards")

    try:
        if len(cards) == 0:
            log_helper.error("话题#{}#评论获取完成")
            break;
        for card in cards:
            blog_card = {}
            if card["card_type"] == 7:
                # blog["is_hotweibo"] = card.get("is_hotweibo",0)
                a = 1
            pic_list = []
            if card["card_type"] == 9:
                blog_card["text_raw"] = card.get("mblog").get("text_raw")
                blog_card["is_hotweibo"] = card.get("is_hotweibo", 0)
                if card.get("mblog").get("source") == "微博视频号":
                    blog_card["is_video"] = 1
                    if card.get("mblog").get("url_struct") is None:
                        blog_card["video_url"] = "获取视频链接失败"
                    else:
                        blog_card["video_url"] = card.get("mblog").get("url_struct")[0].get("short_url")
                    if card.get("mblog").get("page_info") is None:
                        blog_card["video_cover"] = "获取图片信息失败"
                    else:
                        blog_card["video_cover"] = card.get("mblog").get("page_info").get("page_pic")
                else:
                    if card.get("mblog").get("url_struct") is not None:
                        long_url = card.get("mblog").get("url_struct")[0].get("long_url")
                        if long_url.find("video.weibo.com") != -1:
                            blog_card["is_video"] = 1
                            blog_card["video_url"] = long_url
                            blog_card["video_cover"] = card.get("mblog").get("page_info").get("page_pic")
                        else:
                            blog_card["is_video"] = 0
                            blog_card["video_url"] = ""
                            blog_card["video_cover"] = ""
                    else:
                        blog_card["is_video"] = 0
                        blog_card["video_url"] = ""
                        blog_card["video_cover"] = ""

                pic_infos = card.get("mblog").get("pic_infos", {})
                pic_ids = card.get("mblog").get("pic_ids", [])

                for pic_id in pic_ids:
                    if pic_infos.get(pic_id) is None:
                        continue
                    if pic_infos.get(pic_id).get("original") is None:
                        print("original is None")

                    pic_list.append(pic_infos.get(pic_id).get("original").get("url"))

                blog_card["user"] = card.get("mblog").get("user")
                blog_card["pic_list"] = pic_list
                blog_card["create_time"] = card.get("mblog").get("created_at")
            if card["card_type"] == 11:
                if card.get("card_group")[0].get("card_type") == 9:
                    blog_card["is_hotweibo"] = card.get("card_group")[0].get("is_hotweibo", 0)
                    blog_card["text_raw"] = card.get("card_group")[0].get("mblog").get("text_raw")

                    pic_infos = card.get("card_group")[0].get("mblog").get("pic_infos", {})
                    pic_ids = card.get("card_group")[0].get("mblog").get("pic_ids", [])
                    for pic_id in pic_ids:
                        if pic_infos.get(pic_id) is None:
                            continue
                        pic_list.append(pic_infos.get(pic_id).get("original").get("url"))

                    if card.get("card_group")[0].get("mblog").get("source") == "微博视频号":
                        blog_card["is_video"] = 1

                        if card.get("card_group")[0].get("mblog").get("url_struct") is None:
                            blog_card["video_url"] = "获取视频链接失败"
                        else:
                            blog_card["video_url"] = card.get("card_group")[0].get("mblog").get("url_struct")[0].get(
                                "short_url")

                        if card.get("card_group")[0].get("mblog").get("page_info") is None:
                            blog_card["video_cover"] = "获取图片信息失败"
                        else:
                            blog_card["video_cover"] = card.get("card_group")[0].get("mblog").get("page_info").get(
                                "page_pic")
                    else:
                        if card.get("card_group")[0].get("mblog").get("url_struct") is not None:
                            long_url = card.get("card_group")[0].get("mblog").get("url_struct")[0].get("long_url")
                            if long_url.find("video.weibo.com") != -1:
                                blog_card["is_video"] = 1
                                blog_card["video_url"] = long_url
                                blog_card["video_cover"] = card.get("card_group")[0].get("mblog").get("page_info").get("page_pic")
                            else:
                                blog_card["is_video"] = 0
                                blog_card["video_url"] = ""
                                blog_card["video_cover"] = ""
                        else:
                            blog_card["is_video"] = 0
                            blog_card["video_url"] = ""
                            blog_card["video_cover"] = ""
                    blog_card["user"] = card.get("card_group")[0].get("mblog").get("user")
                    blog_card["create_time"] = card.get("card_group")[0].get("mblog").get("created_at")
                    blog_card["pic_list"] = pic_list
                if card.get("card_group")[0].get("card_type") == 10:
                    continue
            if card["card_type"] != 7:
                blog_cards.append(blog_card)
        page += 1
    except Exception as e:
        print(e)




log_helper.error("{} 完成一次爬虫".format(time_str))
